from langchain_community.document_loaders import WebBaseLoader
import bs4


# 设置要获取内容的网页 URL
# url = 'https://zhuanlan.zhihu.com/p/689918127'
# url = 'https://docs.smith.langchain.com/user_guide'
# url = 'https://lilianweng.github.io/posts/2023-06-23-agent/'
# url = 'https://news.sina.com.cn/c/xl/2024-05-27/doc-inawrnhv4026135.shtml'
url = 'https://mp.weixin.qq.com/s/pH6ycN_0zl5giaaRP5shxQ'

# 创建 WebBaseLoader 实例
#loader = WebBaseLoader(web_path=(url,),)

# 抓取微信公众号内容，指定抓取Html的class节点
loader = WebBaseLoader(web_path=(url,),
                       bs_kwargs=dict(
                           parse_only=bs4.SoupStrainer(
                               class_=("rich_media_title", "rich_media_area_primary")
                           )
                       ),
                       )

# 设置用户代理
loader.headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0",
                  "Content-Type": "text/html; charset=UTF-8"}

loader.encoding = 'utf-8'
# 获取网页内容
content = loader.load()

# 打印获取到的内容
print(content)